#coding:utf-8
import scrapy
import json
import unicodedata
import re
import computeDaysByNums
import time

class QuotesSpider(scrapy.Spider):
    name = "commit_nums"

    def start_requests(self):
        #在黑鸭子openhub中进行搜索
        url = 'https://www.openhub.net/'
        tag = getattr(self, 'tag', None)
        if tag is not None:
            url = url + 'p?query=' + tag + '&sort=relevance'
            yield scrapy.Request(url, self.parse_search)

    def parse_search(self, response):
        first_one = response.xpath('//*[@id="project_603413"]/h2/a/@href').extract_first()
        # first_one = response.css('div[id="projects_index_list"] h2[class="title pull-left"] a::attr(href)').extract_first()
        if first_one is not None:
            first_one = response.urljoin(first_one)
            # print first_one
            yield scrapy.Request(first_one, self.parse)

        # for quote in response.css('div.quote'):
        #     yield {
        #         'text': quote.css('span.text::text').extract_first(),
        #         'author': quote.css('small.author::text').extract_first(),
        #     }
        # next_page = response.css('li.next a::attr(href)').extract_first()
        # if next_page is not None:
        #     next_page = response.urljoin(next_page)
        #     yield scrapy.Request(next_page, self.parse)

    def get_commit_nums(self, response):
        #把json转换为python对象
        json_to_python = json.loads(response.body.decode(response.encoding))
        data = json_to_python['series'][0]['data']
        if len(data) > 36:
            data = data[-36:]
        com_per_mon = {}
        for item in data:
            data_str = str(item[0])[0:-3] + '.000'
            # 时间戳转换为时间元组，再格式化输出
            data_num = int(time.strftime("%Y%m", time.localtime(float(data_str))))
            com_per_mon[data_num] = item[1]
        yield com_per_mon

    def parse(self, response):
        code_url = response.css("div[class='chart watermark440']::attr(datasrc)").extract()
        yield scrapy.Request(code_url[1], self.get_commit_nums)
